import json
import math
import pprint as pp
from concurrent.futures import ThreadPoolExecutor

import requests

'''
创建线程池采集案例

采集北京新发地蔬菜价格

何时使用多线程，何时使用多进程
1.多线程：任务相对统一，互相特别相似
2.多进程：多个任务相互独立，很少有交集
'''

f = open('cai.txt', mode='w', encoding='utf-8')

def get_list_info_by_post_data(post_data):
    url = 'http://xinfadi.com.cn/getPriceData.html'
    response = requests.post(url, post_data, headers=headers)
    response.encoding = 'utf-8'
    content = response.json()

    list_info = content['list']
    for li in list_info:
        text1 = 'id:' + str(li['id']) + ' 发布时间:' + li['pubDate'] + ' 品名:' + li['prodName'] + ' 产地:' + li['place']
        text2 = ' 品种:' + li['prodCat'] + ' 最低价：' + li['lowPrice'] + ' 平均价:' + li['avgPrice']
        text3 = ' 最高价:' + li['highPrice'] + ' 单位:' + li['unitInfo']
        text = text1 + text2 + text3 + '\n'
        print(text)

        f.write(text)

if __name__ == '__main__':
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/114.0'}
    url = 'http://xinfadi.com.cn/getPriceData.html'
    post_data = {
                'limit': '20',
                'current': 1,
                'pubDateStartTime': '',
                'pubDateEndTime': '',
                'prodPcatid': '',
                'prodCatid': '',
                'prodName': ''
    }

    # 先获取到一些分页信息
    response = requests.post(url, post_data, headers=headers)
    content = response.json()

    current_page = 1
    max_page = math.ceil(content['count'] / content['limit'])

    # 开启多个线程
    thread_number = 100

    with ThreadPoolExecutor(thread_number) as t:
        while current_page <= 1:
            print('当前采集第 {} 页'.format(current_page))
            post_data = {
                'limit': '20',
                'current': current_page,
                'pubDateStartTime': '',
                'pubDateEndTime': '',
                'prodPcatid': '',
                'prodCatid': '',
                'prodName': ''
            }

            # 多线程采集数据，参数：处理函数的名称，要提交的参数
            t.submit(get_list_info_by_post_data, post_data)

            current_page += 1

f.close()